#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/10/24 14:03
# @Author  : CHEN Wang
# @Site    :
# @File    : factor_analyse.py
# @Software: PyCharm

import pandas as pd
import numpy as np
import scipy.stats as st
import matplotlib.pyplot as plt
import seaborn as sns
import quant_researcher.quant.datasource_fetch.fund_api.api_func_old
from quant_researcher.quant.datasource_fetch.index_api import index_price_related
from quant_researcher.quant.datasource_fetch.fund_api import fund_nav_related
from quant_researcher.quant.datasource_fetch.common_data_api.t_trade_date import \
    get_trade_date_by_mkt
from quant_researcher.quant.datasource_fetch.stock_api import stock_price_related
from quant_researcher.quant.datasource_fetch.factor_api import factor_info, factor_exposure_related
from quant_researcher.quant.datasource_fetch.index_api import index_components
import quant_researcher.quant.factors.factor_preprocess.preprocess as fp
from quant_researcher.quant.factors.factor_composite import multi_factor_process
from quant_researcher.quant.factors.factor_analysis.factor_analyser import plotter
from quant_researcher.quant.performance_attribution.core_functions.performance_analysis.performance import simple_perf
from quant_researcher.quant.project_tool.db_operator import db_conn
from quant_researcher.quant.project_tool.wrapper_tools.common_wrappers import count_time

plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
TK_date_fmt = "%Y-%m-%d"


# def reset_connection():
#     print('数据库连接重建')
#     global CON, CON_factor, CON_astk, CON_ty, CON_wst
#     CON.close()
#     CON_factor.close()
#     CON_astk.close()
#     CON_ty.close()
#     CON_wst.close()
#     # 基础数据组数据库
#     CON = db_conn.create_a_factor_db_conn(by_sqlalchemy=True)
#     # 投研组衍生指标数据库
#     CON_ty = db_conn.get_derivative_factor_db_conn(by_sqlalchemy=True)
#     # 因子数据库
#     CON_factor = db_conn.get_tk_factors_conn(by_sqlalchemy=True)
#     # 维赛特数据库
#     CON_wst = db_conn.get_longbo_conn(by_sqlalchemy=True)
#     CON_astk = db_conn.get_a_stk_conn(by_sqlalchemy=True)


# 定义因子分析类 factorAnalyseMachine
class FactorAnalyseMachine(object):
    def __init__(self, factor_name='', begin_date='', end_date='', asset_type='stock',
                 external_data=False, factor_data=None, direction=[1]):
        """
        初始化因子分析类

        :param factor_name: 分析因子名称，可以是str（单因子），也可以是list（多个因子）
        :param str begin_date: 分析开始时间, '2018-01-01'
        :param str end_date: 分析结束时间, '2020-05-01'
        :param asset_type: 目前asset_type支持'stock', 'manager', 'fund', 'sw', 'zz';
                            'sw' 和'zz' 分别代表申万一级行业和中证一级行业
        :param external_data: bool，是否使用外部因子数据源
        :param factor_data: 外部因子数据，确认数据中至少包含tradedate，资产代码和因子名称三列，
                资产代码为 '资产类型+code'，例如股票为stockcode，基金为fundcode等。
                一个包含ROE指标的数据形状如下：
                —————————————
                stockcode|tradedate   |roe
                000001  |2010-01-03  | 0.2
                000002 |2010-01-03  | 0.15
        :param direction: 因子作用方向, 因子方向正向为1，负向为-1, 默认为正向
        """

        # 基础数据组数据库
        self.con = db_conn.get_basic_data_conn(by_sqlalchemy=True)
        # 因子数据库
        self.con_factor = db_conn.get_tk_factors_conn(by_sqlalchemy=True)
        # 维赛特数据库
        self.con_wst = db_conn.get_third_party_conn(by_sqlalchemy=True)
        # A股数据库
        self.con_astk = db_conn.get_stock_conn(by_sqlalchemy=True)
        self.multi_factor_process = multi_factor_process
        assert end_date > begin_date, "结束日期应大于开始日期"
        self.asset_type = asset_type
        self.external_data = external_data
        if external_data:
            assert factor_data is not None, "factor_data为空，请传入因子数据"
            assert 'tradedate' in factor_data.columns, "请确认数据包含tradedate列"
            assert f'{self.asset_type}code' in factor_data.columns, f"请确认数据包含{self.asset_type}code列"
            self.raw_factor_data = factor_data
            self.begin_date = max(factor_data['tradedate'].min(), begin_date)
            self.end_date = min(factor_data['tradedate'].max(), end_date)
        else:
            self.raw_factor_data = None
            self.begin_date = begin_date
            self.end_date = end_date
        self.direction = direction
        self.factor = factor_name
        self.period_dict = {'daily': 252, 'weekly': 52, 'monthly': 12, 'quarterly': 4}
        # 获取外部因子数据中包含的交易日数据，若使用内部因子则获取begin_date，end_date之间的所有交易日数据
        self.trade_date_list = self.get_trade_date(factor_data)

    def get_trade_date(self, factor_data):
        """
        获取交易日历

        """
        trade_date_df = get_trade_date_by_mkt(start=self.begin_date, end=self.end_date)
        trade_date_list = trade_date_df['trade_date'].tolist()
        if self.external_data:
            unique_date = factor_data['tradedate'].unique()
            mask_1 = (unique_date >= self.begin_date) & (unique_date <= self.end_date)
            mask_2 = pd.Series(unique_date).isin(trade_date_list)
            trade_date_list = unique_date[mask_1 & mask_2].tolist()
        return trade_date_list

    def param_setting(self, period='', universe='HS300', benchmark=None, freq=1,
                      bool_exclude_industry=True, list_exclude_style=[]):
        """
        参数设定方法

        :param str period: 调仓周期
        :param str universe: 标的池仅对股票有效，基金和基金经理对应标的暂定为上证综指
        :param str benchmark: 基准代码，如果输入avg则表示所有资产池的平均值
        :param int freq: 调仓频率
        :param bool bool_exclude_industry: 中性化是否剔除行业因子，仅对股票有效
        :param list list_exclude_style: 中性化是否剔除风格因子，仅对股票有效
        """
        self.period = period
        self.freq = freq
        self.benchmark = benchmark
        self.universe_code_dict = {'SH50': '000016', 'SH180': '000010', 'HS300': '399300',
                                   'ZZ500': '000905', 'ZZ800': '000906', 'CYBZ': '399006',
                                   'ZXB': '399005', 'ALLA': '000985'}
        if self.asset_type in ['stock']:
            self.universe = universe
            self.exclude_industry = bool_exclude_industry
            self.exclude_style = list_exclude_style
            self.universe_code = self.universe_code_dict[self.universe]
        else:
            self.exclude_industry = None
            self.exclude_style = None

    def initial_data(self, **multi_factor_args):
        """
        初始化下载数据，避免多次下载

        :param multi_factor_args: 多因子合成参数，具体参考multi_factor_process的参数设置

        """
        # reset_connection()
        print('获取资产池')
        self.get_asset_pool()
        print('获取资产行情数据')
        self.get_asset_data()
        # reset_connection()
        print('获取因子数据')
        self.get_factor_data()
        print(f'此次分析的是资产类型为: {self.asset_type}; 因子为: {self.raw_factors.tolist()}')
        # reset_connection()
        if self.asset_type == 'stock':
            print('获取股票行业分类数据')
            self.get_stock_industry()
        self.get_dynamic_index_component()
        print('获取基准表现')
        self.get_index_performance()
        print('获取换仓日列表')
        # todo 下面获取调仓时间点的做法可以优化
        time_series = pd.Series(self.trade_date_list).sort_values(ascending=True).reset_index(
            drop=True)
        self.date_list = self.get_rebalance_date(time_series, self.period, self.freq)

        if self.raw_factors.shape[0] > 1:
            if not multi_factor_args:
                print('发现多个因子，请传入多因子参数。')
            print(f'发现{self.raw_factors.shape[0]}个因子：{self.raw_factors.tolist()}, 开始进行因子合成')
            self.factor_data, _ = self.multi_factor_process(factor_data=self.raw_factor_data,
                                                            **multi_factor_args)
            self.factor = self.factor_data.columns.difference(
                ['tradedate', f'{self.asset_type}code', 'manager_code', 'fund_type'])[0]
        else:
            print('Not running multi_factor_process')
            self.factor = self.raw_factors[0]
        print('数据初始化完成')

    def add_factor_direction(self):
        """
        设定因子方向

        :return: 返回方向标记好的因子数据
        """
        if self.asset_type == 'manager':
            fac_data = self.factor_data.set_index(['manager_code', 'fund_type'], append=True).mul(
                np.array(self.direction), axis=1)
        else:
            fac_data = self.factor_data.mul(np.array(self.direction), axis=1)
        return fac_data.reset_index()

    def factor_update(self, factor_name, direction, factor_data=None):
        # reset_connection()
        print('更新因子名称及对应表格')
        if self.factor == factor_name:
            print('已经下载所需因子数据')
            pass
        else:
            if factor_data is None:
                self.factor_data = None
                self.factor = factor_name
                self.direction = direction
                self.factor_table_name, self.factor_name_cn = factor_info.get_factor_table_name(
                    self.factor)
                print('重新获取因子数据')
                self.get_factor_data()
            else:
                self.raw_factor_data = factor_data
                self.factor = factor_name
                self.direction = direction
                self.begin_date = factor_data['tradedate'].min()
                self.end_date = factor_data['tradedate'].max()
                self.trade_date_list = self.get_trade_date(factor_data)
                self.get_factor_data()

    def get_asset_pool(self):
        """
        获取资产池，股票资产从数据库获取，其他类别资产从外部数据获取

        """

        if self.external_data:
            external_pool = self.raw_factor_data[f'{self.asset_type}code'].unique().tolist()
            if self.asset_type != 'stock':
                self.asset_pool = external_pool
            else:
                universe_pool = index_components.get_index_hist_stock_pool(
                    self.universe_code,
                    self.begin_date, self.end_date)['stock_code'].tolist()
                if len(external_pool) < len(universe_pool):
                    print("提供的因子池股票数量小于所选股票池股票数量，可能存在问题。将以因子股票池重新定义资产池")
                    self.asset_pool = external_pool
                else:
                    self.asset_pool = universe_pool

        else:
            self.asset_pool = index_components.get_index_hist_stock_pool(
                self.universe_code,
                self.begin_date, self.end_date)['stock_code'].tolist()

    def get_factor_data(self):
        """
        获取因子数据

        """
        if self.raw_factor_data is not None:
            mask = (self.raw_factor_data['tradedate'] >= self.begin_date) & (
                    self.raw_factor_data['tradedate'] <= self.end_date) & \
                   (self.raw_factor_data['tradedate'].isin(self.trade_date_list))

            if self.asset_type == 'manager':
                self.mgr_match = \
                    self.raw_factor_data.drop_duplicates(['manager_code', 'fund_type'])[
                        ['manager_code', 'fund_type']]
                self.mgr_match[f'{self.asset_type}code'] = np.arange(len(self.mgr_match))
                self.factor_data = \
                    self.raw_factor_data.merge(self.mgr_match, on=['manager_code', 'fund_type'])[
                        mask].set_index(
                        ['tradedate', f'{self.asset_type}code'])
            else:
                self.factor_data = self.raw_factor_data[mask].set_index(
                    ['tradedate', f'{self.asset_type}code'])
            self.raw_factors = self.factor_data.columns.difference(['manager_code', 'fund_type'])
            self.factor_data = self.add_factor_direction()
        else:
            if self.asset_type == 'stock':
                self.raw_factors = np.array([self.factor])
                self.factor_table_name, self.factor_name_cn = factor_info.get_factor_table_name(
                    self.factor)
                # factor_data = factor_exposure_related.get_stock_factor_exposure(self.factor, self.asset_pool,
                #                                                                 self.begin_date, self.end_date)
                factor_data = factor_exposure_related.get_stock_factor_exposure(self.factor, self.asset_pool,
                                                                                self.begin_date, self.end_date)
                factor_data = factor_data.rename(columns={'end_date': 'tradedate', 'stock_code': 'stockcode'})
                factor_data.dropna(inplace=True)
                factor_data = factor_data.set_index(['tradedate', 'stockcode'])[
                    self.factor].unstack().reindex(self.trade_date_list).ffill().stack().rename(
                    self.factor).reset_index()
                factor_data.columns = ['tradedate', 'stockcode', self.factor]
                factor_data['tradedate'] = factor_data['tradedate'].astype('str')
                self.factor_data = factor_data.reset_index(drop=True)
                # 加上股票因子的方向
                self.factor_data.set_index(['tradedate', 'stockcode'], inplace=True)
                self.factor_data = self.add_factor_direction()
                # print('获取股票行业分类数据')
                self.get_stock_industry()
        assert self.raw_factors.shape[0] == len(
            self.direction), f'请确认标记所有因子的方向，共{self.raw_factors.shape[0]}个因子，已标记' f'{len(self.direction)}个。'

    def get_dynamic_index_component(self):
        """
        获取股票池动态成分股数据， 目前支持'000016', '000010', '399300', '000905', '399006', '399905' 股票池

        """
        if not self.external_data:
            if self.universe not in self.universe_code_dict.keys():
                print(u'暂不支持此类型的资产池')
                return
            temp = index_components.get_index_dynamic_stock_pool(self.universe_code,
                                                                 self.begin_date, self.end_date)
            temp = temp.rename(columns={'trade_date': 'tradedate', 'stock_code': 'stockcode'})
            ticker_list = temp[['tradedate', 'stockcode']]
            ticker_list['tradedate'] = pd.to_datetime(ticker_list['tradedate']).dt.strftime(
                '%Y-%m-%d')
            self.stock_pool_group = ticker_list.groupby('tradedate')
        else:
            self.stock_pool_group = self.factor_data.groupby('tradedate')[
                [f'{self.asset_type}code', 'tradedate']]

    def get_asset_data(self):
        """
        获取资产行情数据，目前有stock，fund，manager，zz，sw几个类别

        """
        if self.asset_type == 'stock':
            allPrice = stock_price_related.get_stock_quote(self.asset_pool, self.begin_date, self.end_date,
                                                           select=['stock_code', 'end_date',
                                                                      'high', 'low', 'close as fap_close'])
            allPrice = allPrice.rename(columns={'stock_code': 'stockcode', 'end_date': 'tradedate'})
        elif self.asset_type == 'fund':
            # allPrice = fund_nav_related.get_fund_adj_nav(self.asset_pool, [('nav_date', self.begin_date, self.end_date)],
            #                                              return_only_nav=False)
            allPrice = fund_nav_related.get_fund_nav(fund_code=self.asset_pool, start_date=self.begin_date, end_date=self.end_date)
            allPrice['end_date'] = pd.to_datetime(allPrice['end_date'])
            allPrice['end_date'] = allPrice['end_date'].dt.date
            allPrice = allPrice.rename(columns={'end_date': 'tradedate', 'fund_code': 'fundcode', 'adjusted_nav': 'fap_close'})
        elif self.asset_type == 'manager':
            # allPrice1 = fund_nav_related.get_manager_adj_nav(start=self.begin_date, end=self.end_date)
            allPrice = fund_nav_related.get_manager_nav(
                start_date=self.begin_date,
                end_date=self.end_date,
                fund_type='00'
            )
            allPrice['end_date'] = pd.to_datetime(allPrice['end_date'])
            allPrice['end_date'] = allPrice['end_date'].dt.date
            allPrice = allPrice.rename(columns={'end_date': 'tradedate', 'adjusted_nav': 'fap_close'})
            allPrice = allPrice.merge(self.mgr_match, on=['manager_code', 'fund_type'])
        elif self.asset_type in ['zz', 'sw']:
            # allPrice = index_price_related.get_stock_index_quote(self.asset_pool, self.begin_date, self.end_date,
            #                                                      select=['trade_date as tradedate',
            #                                                              'index_code as industrycode',
            #                                                              'close as fap_close', 'high', 'low'])
            allPrice = index_price_related.get_stock_index_quote(self.asset_pool, self.begin_date, self.end_date,
                                                                 select=['trade_date as tradedate',
                                                                         f'index_code as {self.asset_type}code',
                                                                         'close as fap_close', 'high', 'low'])
        allPrice['tradedate'] = pd.to_datetime(allPrice['tradedate']).dt.strftime(TK_date_fmt)
        # 对齐交易日历
        allPrice = allPrice.set_index(['tradedate', f'{self.asset_type}code'])['fap_close']
        allPrice = allPrice.unstack().reindex(self.trade_date_list).stack().rename(
            'fap_close').reset_index()
        allPrice[f'{self.asset_type}code'] = allPrice[f'{self.asset_type}code'].astype(str)
        self.all_stock_price = allPrice
        self.all_stock_price_group = self.all_stock_price.groupby('tradedate')

    def get_index_performance(self):
        """
        获取基准表现

        """
        if self.benchmark is None:
            if self.asset_type == 'stock':
                index_code = self.universe_code
            else:
                index_code = '000001'
        else:
            index_code = self.benchmark
        if index_code == 'avg':
            index_df = self.all_stock_price.set_index(['tradedate', f'{self.asset_type}code'])[
                'fap_close'].unstack().ffill().pct_change().mean(axis=1)
            self.index_return = index_df
        else:
            # index_df1 = index_price_related.get_index_close(index_code, self.begin_date, self.end_date).reset_index()
            index_df = index_price_related.get_index_close(
                index_code=index_code,
                start_date=self.begin_date,
                end_date=self.end_date
            ).rename(columns={'end_date': 'trade_date'}).set_index(['trade_date', 'index_code'])['close'].unstack()
            index_df.index = pd.to_datetime(index_df.index)
            index_df = index_df.reset_index()
            index_df = index_df.rename(columns={'trade_date': 'tradedate', 'index_code': 'indexcode'})
            index_df['tradedate'] = pd.to_datetime(index_df.tradedate).dt.strftime('%Y-%m-%d')
            index_df = index_df.set_index('tradedate')
            self.index_return = index_df.ffill().pct_change().iloc[:, 0]

    def get_stock_industry(self):
        """
        获取股票的行业分类信息，以enddate为准

        """
        # 取enddate的行业分类信息
        industry = pd.read_sql(f"select stockcode, sw_ind1 from stk_fac_ind_sw_ind1 "
                               f"where tradedate = '{self.end_date}'", self.con_factor)
        self.industry_information = industry

    # ********************************************
    # 初识因子
    # ********************************************
    def describe_factor(self, date='', winsor=True, neutral=True, standard=True, plot=True):
        """
        考察universe内股票在tradedate这天的factor值分布情况，以及factor与基准因子风格的相关性情况
        输入date为指定日期，如果没有指定日期，默认用self中的date

        :param date: 指定日期
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param plot: 是否作图
        :return:
        """

        print('本次关注的因子是：%s' % self.factor)
        print('=' * 100, u'\n', '(%s)' % self.factor, '的分布情况是：')

        if date == '':
            date = self.trade_date_list[0]
        df = self.factor_data[self.factor_data['tradedate'] == date][
            [self.factor, f'{self.asset_type}code']]
        # df = pd.DataFrame(temp, columns=column_names)
        df.sort_values(f'{self.asset_type}code', inplace=True)
        df.set_index(f'{self.asset_type}code', inplace=True)
        # univevse_data = set_universe(self.universe, date)
        if self.asset_type == 'stock':
            universe_data = self.stock_pool_group.get_group(date)
            universe = universe_data['stockcode']
            df = df[df.index.isin(universe)]
        # 去极值，中性化标准化等
        df = fp.preprocess(df, self.factor, self.asset_type, date, winsor, neutral, standard,
                           self.exclude_industry, self.exclude_style)
        df.reset_index(inplace=True)
        if plot:
            sns.distplot(df[self.factor].dropna().astype(float), hist=True, kde=False,
                         hist_kws={"rwidth": 0.90})
            plt.title(self.factor + '因子分布')
            plt.show()
        #
        # timec1 = dt.datetime.now()
        # conn = pymssql.connect(host='alpha.atrader.com.cn:11433', user='QRUser04', password='Passw0rdBP',
        #                        database='BPFactor')
        # cursor = conn.cursor()
        # cursor.execute(
        #     "SELECT stockcode,beta,momentum,size,earnyild,resvol,growth,btop,leverage,liquidty,sizenl from BPRM_exposure where tradedate = '%s'" % (
        #         date))
        # temp = cursor.fetchall()
        # column_names = [item[0] for item in cursor.description]
        #
        # style_factor = pd.DataFrame(temp, columns=column_names)
        # corrDict = {}
        # corrDf = pd.merge(df, style_factor, on='stockcode', how='inner').dropna(axis='index')
        # for x in style_factor.columns[1:]:
        #     corrDict[x] = st.pearsonr(corrDf[self.factor], corrDf[x])
        #
        # pearsonDf = pd.DataFrame(corrDict, index=['r', 'p']).T
        # print('=' * 100, u'\n', '在', self.universe, '中，', '(%s)' % self.factor, '与常用风格因子池的相关性为：')
        # self._corrDisplayFactorBenchmark(pearsonDf)
        # timec2 = dt.datetime.now()
        # print('风格因子相关系数耗时：', timec2 - timec1)

        ###########################

        # return corrDict

    def factor_ind_dist(self, date, winsor=True, neutral=True, standard=True, plot=True):
        """
        给出因子在指定日期时在各个行业的分布情况，返回列名为Q20,Q50,Q80的DataFrame

        :param date: 指定日期，默认为第一个交易日
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param plot: 是否做图
        :return: 返回因子行业分布的dataframe
        """
        if date == '':
            date = self.trade_date_list[0]
        res = {}

        industry = self.industry_information.copy()
        industryDict = np.unique(industry['sw_ind1'])
        merge_data = pd.merge(self.factor_data, industry, on='stockcode', how='inner')
        for x in industryDict:
            tmpSeries = merge_data[merge_data['sw_ind1'] == x]
            tmpSeries.set_index('stockcode', inplace=True)
            if len(tmpSeries) == 0:
                res[x] = [np.nan] * 3
                continue
            tmpSeries = fp.preprocess(tmpSeries, self.factor, self.asset_type, date, winsor,
                                      neutral, standard, self.exclude_industry, self.exclude_style)
            tmpSeries.dropna(inplace=True)
            res[x] = [tmpSeries[self.factor].quantile(0.2), tmpSeries[self.factor].quantile(0.5),
                      tmpSeries[self.factor].quantile(0.8)]
        industryDist = pd.DataFrame(res, index=['Q20', 'Q50', 'Q80']).T
        if plot:
            plotter.industry_dist_plot(industryDist, self.factor)
        return industryDist

    # *****************************************
    # IC / Rank IC - 关注盈利预测能力
    # *****************************************
    def ic_calc(self, factor, factor_data, beginDate='', endDate='', winsor=True, neutral=False,
                standard=True,
                rank=True):
        """
        计算给定因子股票池时，tradedate前samplePeriod天因子均值与之后predictPeriod天前瞻收益率的相关系数与p值,
        如果rank为True，返回秩相关系数，否则相关系数

        :param factor: 因子名称
        :param factor_data: 因子数据
        :param beginDate: 开始日期
        :param endDate: 结束日期，如果无输入endDate，先获取从开始日期至今的交易日，
        取第period个时间为endDate
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param rank: 是否为秩相关系数
        :return:返回IC值
        """

        Univ = self.stock_pool_group.get_group(beginDate)
        # 获得起始价格
        beginPrice = self.all_stock_price[self.all_stock_price['tradedate'] == beginDate]
        beginPrice = pd.merge(beginPrice, Univ, on=f'{self.asset_type}code', how='inner')[
            [f'{self.asset_type}code', 'fap_close']]

        # 获取终止价格
        endPrice = self.all_stock_price[self.all_stock_price['tradedate'] == endDate]
        endPrice = pd.merge(endPrice, Univ, on=f'{self.asset_type}code', how='inner')[
            [f'{self.asset_type}code', 'fap_close']]

        Price = pd.merge(beginPrice, endPrice, on=f'{self.asset_type}code', how='inner')
        Price['return'] = (Price['fap_close_y'] / Price['fap_close_x'] - 1).T
        Price['return'] = Price['return'].astype(float)

        temp = factor_data[(factor_data['tradedate'] == beginDate)]  # 取出所选当天
        factor_mean_data = temp.set_index(f'{self.asset_type}code')
        # 若所选当天没有相关因子数据 则返回0
        if len(factor_mean_data) == 0:
            return 0
        self.factor_mean_data = factor_mean_data
        factor_mean_data = fp.preprocess(factor_mean_data, factor, self.asset_type,
                                         beginDate, winsor, neutral, standard,
                                         self.exclude_industry, self.exclude_style)
        factor_mean_data.reset_index(inplace=True)
        Price = pd.merge(Price, factor_mean_data, on=f'{self.asset_type}code', how='inner')
        Price = Price.dropna()  # rank ic 有nan值时没有结果
        if rank:
            return st.spearmanr(Price[['return', factor]])
        else:
            return st.pearsonr(Price['return'], Price[factor])

    @count_time
    def time_range_ic(self, factor=None, factor_data=None, freq='', period='', winsor=True,
                      neutral=False, standard=True, rank=True, plot=True, figsize=(16, 6)):
        """
        返回一段时间内因子的rank IC值序列，period为因子所预测的下期收益率中间隔得天数

        :param factor: 因子名称
        :param factor_data: 因子数据
        :param freq: 分析频率，默认为调仓频率
        :param period: 分析周期，默认为调仓周期
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param rank: 是否为秩相关系数
        :param plot: 是否作图

        :return: 返回IC时间序列dataframe
        """

        print(f'{self.factor} IC计算开始'.format(factor))
        if period == '':
            period = self.period
        if freq == '':
            freq = self.freq
        if (factor_data is None) or (factor is None):
            factor = self.factor
            factor_data = self.factor_data
        adjusting_date_list = self.control_date_freq(period, freq)  ##根据设置的频率获取交易日期

        factorIC = {}
        for i in range(0, len(adjusting_date_list)):
            if i >= (len(adjusting_date_list) - 1):
                break
            factorIC[adjusting_date_list[i + 1]] = self.ic_calc(factor, factor_data,
                                                                beginDate=adjusting_date_list[i],
                                                                endDate=adjusting_date_list[i + 1],
                                                                winsor=winsor, neutral=neutral,
                                                                standard=standard, rank=rank)
        print('Finished.')

        ICDataFrame = pd.DataFrame(factorIC, index=['r', 'p']).T
        IC_over_zero_ratio = len(ICDataFrame[ICDataFrame.iloc[:, 0] > 0]) / len(ICDataFrame)
        IC_abs_over_ratio = (ICDataFrame.iloc[:, 0].abs() > 0.02).sum() / len(ICDataFrame)
        IC_standard_error = ICDataFrame.iloc[:, 0].std()
        IC_IR_df = ICDataFrame.iloc[:, 0].expanding().mean() / ICDataFrame.iloc[:,
                                                               0].expanding().std()
        IC_IR_ratio = IC_IR_df.iloc[-1]
        IC_mean = ICDataFrame.iloc[:, 0].mean()
        IC_information = pd.DataFrame(
            {'因子英文名': factor, '总体IC均值': IC_mean, '总体IC大于0比率': IC_over_zero_ratio, 'IC标准差': \
                IC_standard_error, '总体IC_IR': IC_IR_ratio, '总体IC绝对值大于0.02比率': IC_abs_over_ratio},
            index=[0])
        self.ic_information = IC_information
        print(IC_information)
        if plot:
            plotter.ic_time_plot(ICDataFrame, figsize)
        self.IC = ICDataFrame
        return ICDataFrame, IC_IR_df, IC_information

    def get_rebalance_date(self, date_series, _period, freq=''):
        time = pd.to_datetime(date_series)
        time_table = pd.DataFrame()
        time_table['tradedate'] = time
        time_table['year'] = time_table['tradedate'].dt.year
        time_table['quarter'] = time_table['tradedate'].apply(
            lambda x: (int(x.strftime('%m')) - 1) // 3 + 1)
        time_table['month'] = time_table['tradedate'].dt.month
        time_table['week'] = time_table['tradedate'].dt.strftime('%W')

        if _period == 'daily':
            temp_date_list = time.dt.strftime('%Y-%m-%d').unique().tolist()
        elif _period == 'weekly':
            temp_date_list = time_table[time_table['week'] != time_table['week'].shift(1)][
                'tradedate'].dt.strftime(
                '%Y-%m-%d').tolist()
        elif _period == 'monthly':
            temp_date_list = time_table[time_table['month'] != time_table['month'].shift(1)][
                'tradedate'].dt.strftime('%Y-%m-%d').tolist()
        elif _period == 'quarterly':
            temp_date_list = \
                time_table[time_table['quarter'] != time_table['quarter'].shift(1)][
                    'tradedate'].dt.strftime('%Y-%m-%d').tolist()
        date_list = temp_date_list[:len(temp_date_list):freq]
        # 如果整个时间区间的最后一天同时也是调仓日，则不进行调仓
        if self.trade_date_list[-1] == date_list[-1]:
            date_list = date_list[:-1]
        return date_list

    def control_date_freq(self, _period, freq='', turnover=False):
        """
        如果换仓频率和初始化参数一致，返回初始化过程的日期列表，如果不一致，重新算日期列表

        :param _period: 换仓周期
        :param freq: 换仓频率
        :return:
        """
        # if (self.period == _period) and (not turnover):
        #     date_list = self.date_list
        # else:
        # 暂时强制重算调仓日期
        date_list = self.get_rebalance_date(self.factor_data['tradedate'], _period, freq)
        return date_list

    @count_time
    def longshort_portfolio(self, period='', freq='', winsor=True, neutral=False, standard=True,
                            rank=True, abnormal_threshold=0.11, group_num=5, cut_type='quantile', weight_method='等权',
                            plot=True, figsize=(16, 12)):
        """
        因子分层回测函数，展示不同组别的净值曲线

        :param period: 调仓周期
        :param freq: 调仓频率，如period为月，freq为3，则为3个月调仓
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param rank: 是否为秩相关系数
        :param abnormal_threshold: 行情数据异常值阈值
        :param group_num: 分组数目
        :param weight_method: 加权方式，支持等权，市值加权，价格加权
        :param plot: 是否做图
        :param figsize: 作图大小
        :return:
        """
        print('因子多空组合')
        if period == '':
            period = self.period
        if freq == '':
            freq = self.freq
        if weight_method in ['流通市值加权', '总市值加权', '市值加权', '价格加权']:
            assert self.asset_type == 'stock', "只有股票资产支持此类加权方式"
            if weight_method in ['市值加权', '流通市值加权']:
                mkt_value = quant_researcher.quant.datasource_fetch.fund_api.api_func_old.get_stock_mkv(
                    self.begin_date, self.end_date,
                    self.asset_pool, self.con_astk)
                mkt_value = mkt_value.stack().reset_index()
                mkt_value.columns = ['tradedate', 'stockcode', 'float_mkv']
                mkt_value.tradedate = mkt_value.tradedate.dt.strftime(TK_date_fmt)
            elif weight_method == '总市值加权':
                mkt_value = quant_researcher.quant.datasource_fetch.fund_api.api_func_old.get_stock_mkv(
                    self.begin_date, self.end_date,
                    self.asset_pool, self.con_astk,
                    mkv_type='总市值')
                mkt_value = mkt_value.stack().reset_index()
                mkt_value.columns = ['tradedate', 'stockcode', 'float_mkv']
                mkt_value.tradedate = mkt_value.tradedate.dt.strftime(TK_date_fmt)
        distinct_trade_date = pd.Series(self.all_stock_price['tradedate'][self.all_stock_price['tradedate'].isin(self.trade_date_list)].unique().tolist())
        time_length = len(distinct_trade_date[1:])
        adjusting_date_list = self.control_date_freq(period, freq)
        allPrice_group = self.all_stock_price.groupby('tradedate')
        factor_data_group = self.factor_data.groupby('tradedate')
        allreturn = self.all_stock_price.set_index(['tradedate', f'{self.asset_type}code'])[
            'fap_close'].unstack().pct_change()
        #  对于创业板和科创板的股票设置20%的涨跌幅限制，其他版块10%的限制。以此来剔除异常收益率。
        hi_tech_sector = allreturn.columns.str[:3].isin(['300','688'])
        hi_tech_sector_return = allreturn.loc[:, hi_tech_sector]
        allreturn.loc[:, hi_tech_sector] = allreturn.loc[:, hi_tech_sector].where(hi_tech_sector_return.abs()<=0.21, 0)
        allreturn.loc[:, ~hi_tech_sector] = allreturn.loc[:, ~hi_tech_sector].where(allreturn.loc[:, ~hi_tech_sector].abs()<abnormal_threshold,0)
        allreturn = allreturn.stack().reset_index()
        allreturn.rename(columns={0: 'return'}, inplace=True)
        allreturn_group = allreturn.groupby('tradedate')

        group_return = [[] for xx in range(group_num)]
        allICP = pd.DataFrame()
        date_record = []
        for datenum in range(0, len(adjusting_date_list)):  # 每period天等权调一次仓
            '''计算多空收益'''
            # 每次调仓获取更新的资产池
            try:
                Univ = self.stock_pool_group.get_group(adjusting_date_list[datenum])
            except:
                print(f'Missed data {adjusting_date_list[datenum]} in longshortPortfolio')
                continue
            print('调仓日期：', adjusting_date_list[datenum])
            # 取出当天数据
            beginPrice = allPrice_group.get_group(adjusting_date_list[datenum])
            factor_data = factor_data_group.get_group(adjusting_date_list[datenum])
            beginPriceFactor = beginPrice.merge(
                Univ, on=[f'{self.asset_type}code', 'tradedate']).merge(
                factor_data, on=[f'{self.asset_type}code', 'tradedate'])
            # TODO 处理因子数据缺失问题
            # if beginPriceFactor.shape[0] < group_num:
            #     print('股票数目小于分组数目，跳过')
            #     continue
            beginPriceFactor.drop_duplicates(inplace=True)
            beginPriceFactor = beginPriceFactor.set_index(f'{self.asset_type}code')
            beginPriceFactor = fp.preprocess(beginPriceFactor, self.factor, self.asset_type,
                                             distinct_trade_date[datenum], winsor, neutral,
                                             standard, self.exclude_industry, self.exclude_style)
            beginPriceFactor.reset_index(inplace=True)
            beginPriceFactor['quantile'] = fp.my_qcut2(beginPriceFactor[self.factor].values,
                                                       group_num, cut_type=cut_type)
            ##分组数可以作为系数
            temp_code_list = []
            for turni in range(group_num):
                temp_code_list.append(
                    beginPriceFactor[beginPriceFactor['quantile'] == turni + 1][
                        f'{self.asset_type}code'].tolist())
            if datenum == len(adjusting_date_list) - 1:
                self.final_candidate = temp_code_list
            ##本次调仓的结束日期
            enddate = datenum + 1
            ##如果结束日期大于数据长度，设为最后一天
            if enddate > (len(adjusting_date_list) - 1):
                temp_daily_date_list = distinct_trade_date[
                    distinct_trade_date >= adjusting_date_list[datenum]].tolist()
            elif enddate <= (len(adjusting_date_list) - 1):
                temp_daily_date_list = distinct_trade_date[
                    (distinct_trade_date >= adjusting_date_list[datenum]) \
                    & (distinct_trade_date <= adjusting_date_list[enddate])].tolist()

            ######计算多空收益
            # #获取区间时间列表
            for date in temp_daily_date_list[1:]:
                dayreturn = allreturn_group.get_group(date)
                temp_group_return = []
                for turni in range(group_num):
                    return_df = dayreturn[
                        dayreturn[f'{self.asset_type}code'].isin(temp_code_list[turni])]
                    if weight_method == '市值加权':
                        day_mkv_value = mkt_value[mkt_value['tradedate'] == date]
                        return_df = return_df.merge(day_mkv_value, on=['tradedate', 'stockcode'],
                                                    how='left')
                        weight = return_df['float_mkv'] / return_df['float_mkv'].sum()
                    elif weight_method == '价格加权':
                        return_df = return_df.merge(self.all_stock_price,
                                                    on=['tradedate', 'stockcode'], how='left')
                        weight = return_df['fap_close'] / return_df['fap_close'].sum()
                    else:
                        weight = 1 / return_df.shape[0] if return_df.shape[0] != 0 else 0
                    temp_group_return.append((return_df['return'] * weight).sum())
                [group_return[idx].append(value) for idx, value in enumerate(temp_group_return)]
                date_record.append(date)
                # 如果是换仓最后一天，计算期间收益
                if date == temp_daily_date_list[-1]:
                    begin_endPriceFactor = beginPriceFactor.merge(
                        allPrice_group.get_group(temp_daily_date_list[-1]),
                        on=f'{self.asset_type}code', how='left')
                    begin_endPriceFactor['return'] = (begin_endPriceFactor['fap_close_y'] /
                                                      begin_endPriceFactor[
                                                          'fap_close_x']) - 1
                    icdata = begin_endPriceFactor.groupby('quantile').apply(
                        lambda x: st.spearmanr(
                            x[['return', self.factor]].astype(float)) if rank else st.pearsonr(
                            x['return'].astype(float), x[self.factor].astype(float)))
                    icdata2 = list(map(list, icdata))

                    ICP = pd.DataFrame(icdata2,
                                       index=np.repeat(temp_daily_date_list[-1], group_num),
                                       columns=['IC', 'P-value'])
                    ICP['quantile'] = icdata.index.tolist()
                    allICP = pd.concat([allICP, ICP])
        group_return_df = pd.DataFrame(group_return).T
        group_return_df.index = distinct_trade_date[1:]
        group_return_df.columns = icdata.index.tolist()
        # 年收益统计
        return_analysis = group_return_df.copy()
        # return_analysis=(return_analysis+1).cumprod()
        return_analysis.reset_index(inplace=True)
        columns_name = []
        columns_name.append('date')
        for name_i in range(group_num):
            columns_name.append('Q' + str(name_i + 1))
        return_analysis.columns = columns_name
        return_analysis['date'] = pd.to_datetime(return_analysis['date'])

        allyear_return = (return_analysis.set_index('date') + 1).cumprod().resample('1y').last()
        year = return_analysis['date'].apply(lambda x: x.year).unique().tolist()
        fake_begin_df = pd.DataFrame([np.repeat(1, group_num)], columns=columns_name[1:])
        fake_begin_df['date'] = pd.to_datetime(str(year[0]) + '-01-01')
        fake_begin_df.set_index('date', inplace=True)
        allyear_return = allyear_return.append(fake_begin_df).sort_index().pct_change(
            1).dropna()  ##每年收益
        ##月收益
        return_montly = (
            (return_analysis[return_analysis['date'].apply(lambda x: x.year) == year[-1]].set_index(
                'date') + 1).cumprod()).resample('1M').last()
        fake_begin_monthly = pd.DataFrame([np.repeat(1, group_num)], columns=columns_name[1:])
        fake_begin_monthly['date'] = pd.to_datetime(str(year[-1]) + '-01-01')
        fake_begin_monthly.set_index('date', inplace=True)
        return_montly = return_montly.append(fake_begin_monthly).sort_index().pct_change(1).dropna()
        ##总

        temp_value = ((allyear_return + 1).prod() - 1).tolist()
        temp_value.insert(0, '长期')
        temp_columns = columns_name[1:]
        temp_columns.insert(0, 'date')
        total_return = pd.DataFrame([temp_value], columns=temp_columns)

        allyear_return.reset_index(inplace=True)
        allyear_return['date'] = allyear_return['date'].apply(lambda x: str(x.year))
        return_montly.reset_index(inplace=True)
        return_montly['date'] = return_montly['date'].apply(
            lambda x: str(x.year) + '-' + str(x.month))
        temp_one = pd.concat([allyear_return, return_montly])
        final_result = pd.concat([temp_one, total_return])
        final_result['long_short'] = final_result[columns_name[-1]] - final_result['Q1']

        # 计算因子的均值，标准差，IR
        ICmean = allICP.groupby(['quantile']).apply(lambda x: x['IC'].mean())

        # Pmean = allICP.groupby(['quantile']).apply(lambda x: x['P-value'].mean())
        IC_larger_zero_ratio = allICP.groupby(['quantile']).apply(
            lambda x: len(x[x['IC'] > 0]) / len(x))
        ICstd = allICP.groupby(['quantile']).apply(lambda x: x['IC'].std())

        IC_IR = allICP.groupby(['quantile']).apply(lambda x: x['IC'].mean()) / ICstd

        group_return_df = group_return_df.astype(float)
        Annualized_return = group_return_df.apply(
            lambda x: ((x + 1).prod()) ** (252 / time_length) - 1)
        Annualized_std = group_return_df.apply(lambda x: x.std() * (252 ** (1 / 2)))

        Annualized_group_IR = group_return_df.apply(
            lambda x: ((x - self.index_return.astype(float) + 1).prod() ** (
                    252 / time_length) - 1) / \
                      ((x - self.index_return.astype(float)).std() * (252 ** (1 / 2)))
        )

        long_short_return = (pd.Series(group_return[-1]) - pd.Series(group_return[0]))
        index_return = self.index_return
        index_trade_date = list(index_return.index)
        long_short_return.index = distinct_trade_date[1:]
        long_short_return = long_short_return.reindex(index_return.index)
        Annualized_longshort = (long_short_return + 1).prod() ** (252 / time_length) - 1
        Annualized_longshort_std = long_short_return.std() * (252 ** (1 / 2))
        Annualized_longshort_IR = (((long_short_return - index_return) + 1).prod() ** (
                252 / time_length) - 1) / \
                                  ((long_short_return - index_return).std() * (
                                          252 ** (1 / 2)))

        # 生成大表
        ICstatsdf = pd.DataFrame([ICmean, IC_larger_zero_ratio, ICstd, IC_IR, Annualized_return,
                                  Annualized_std, Annualized_group_IR],
                                 index=['IC均值', 'IC大于0所占比', 'IC标准差', 'IC_IR', '年化收益', '年化标准差',
                                        '信息比率']).T
        long_short_df = pd.DataFrame(
            {'多空组合年化收益': Annualized_longshort, '多空组合年化标准差': Annualized_longshort_std,
             '信息比率': Annualized_longshort_IR},
            index=['long_short_group'])

        group_ret_dict = dict()
        active_group_ret_dict = dict()
        sharpe_dict = dict()
        maxdraw_dict = dict()
        ann_dict = dict()
        active_dict = dict()
        track_error_dict = dict()
        volatility_dict = dict()
        ir_dict = dict()
        win_rate = dict()
        index_perf = simple_perf(index_return)
        for ii in range(group_num):
            label = '分组' + str(ii + 1)
            group_ret = pd.Series(group_return[ii], index=distinct_trade_date[1:]).reindex(
                index_trade_date)
            cumulated_active_group_ret = (group_ret + 1).cumprod() - (index_return + 1).cumprod()
            active_group_ret = group_ret - index_return
            track_error = active_group_ret.std()
            ir = active_group_ret.mean() / track_error
            group_perf = simple_perf(group_ret)
            ann_active = group_perf['年化收益率'] - index_perf['年化收益率']
            sharpe_dict[label] = group_perf['夏普比率']
            maxdraw_dict[label] = group_perf['最大回撤']
            ann_dict[label] = group_perf['年化收益率']
            volatility_dict[label] = group_perf['年化波动率']
            active_dict[label] = ann_active
            group_ret_dict[label] = group_ret
            track_error_dict[label] = track_error * np.sqrt(252)
            win_rate[label] = (active_group_ret > 0).sum() / active_group_ret.count()
            ir_dict[label] = ir
            active_group_ret_dict[label] = cumulated_active_group_ret
        sharpe_df = pd.DataFrame.from_dict(sharpe_dict, 'index', columns=['夏普比率'])
        maxdraw_df = pd.DataFrame.from_dict(maxdraw_dict, 'index', columns=['最大回撤'])
        track_error_df = pd.DataFrame.from_dict(track_error_dict, 'index', columns=['跟踪误差'])
        volatility_dict_df = pd.DataFrame.from_dict(volatility_dict, 'index', columns=['年化波动率'])
        ir_df = pd.DataFrame.from_dict(ir_dict, 'index', columns=['信息比率'])
        ann_df = pd.DataFrame.from_dict(ann_dict, 'index', columns=['年化收益率'])
        ann_active_df = pd.DataFrame.from_dict(active_dict, 'index', columns=['年化超额收益率'])
        win_rate_df = pd.DataFrame.from_dict(win_rate, 'index', columns=['相对基准胜率'])
        performance = pd.concat(
            [ann_df, volatility_dict_df, sharpe_df, -maxdraw_df, track_error_df, ir_df,
             ann_active_df, win_rate_df], axis=1)
        performance = pd.concat([performance, index_perf.rename('benchmark').to_frame().T])
        excess_return = (pd.Series(group_return[-1]) + 1).cumprod() - (
                pd.Series(group_return[0]) + 1).cumprod()
        excess_return.index = distinct_trade_date[1:]
        excess_return = excess_return.reindex(index_return.index)
        group_return_df['benchmark'] = index_return
        if plot:
            plotter.group_plot(self.factor, active_group_ret_dict, group_ret_dict, index_return,
                               excess_return, figsize)
        print(final_result)
        print(performance)
        return ICstatsdf, long_short_df, performance, group_return_df

    @count_time
    def factor_turnover_simple(self, freq='', period='', winsor=False, neutral=False, standard=True,
                               group_num=5, cut_type='quantile', plot=True):
        """
        多空体系算turnover，分别计算每组的换手率

        :param freq: 调仓频率
        :param period: 调仓周期
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param group_num: 分组数目
        :return:
        """
        if period == '':
            period = self.period
        if freq == '':
            freq = self.freq
        factor_data = self.factor_data.copy()
        # factor_data = factor_data.set_index(['tradedate','stockcode'])[self.factor].unstack().reindex(self.trade_date_list).ffill().stack().rename(self.factor).reset_index()
        # factor_data.columns = ['tradedate', 'stockcode', self.factor]
        factor_data_group = factor_data.groupby('tradedate')
        # factor_data['tradedate'] = factor_data['tradedate']

        adjusting_date_list = self.control_date_freq(period, freq, turnover=False)

        daterecord = []
        turn_over_store_list = [[] for xx in range(group_num)]

        for datenum in range(0, len(adjusting_date_list)):
            '''换手率'''
            Univ = self.stock_pool_group.get_group(adjusting_date_list[datenum])

            # print('换仓日期:', adjusting_date_list[datenum])
            # 取出当天数据
            # beginPrice=allPrice_group.get_group(distinct_trade_date[datenum])
            try:
                factor_data_day = factor_data_group.get_group(adjusting_date_list[datenum])
            except:
                print('因子数据缺失，跳过')
                continue

            # 限定资产池
            factor_merge = factor_data_day.merge(Univ, how='inner', on=f'{self.asset_type}code')

            factor_merge = factor_merge.set_index(f'{self.asset_type}code')
            ####
            factor_merge = fp.preprocess(factor_merge, self.factor, self.asset_type,
                                         adjusting_date_list[datenum], winsor, neutral,
                                         standard, self.exclude_industry, self.exclude_style)
            factor_merge.reset_index(inplace=True)
            ###
            factor_merge['quantile'] = fp.my_qcut2(factor_merge[self.factor], group_num, cut_type=cut_type)

            temp_code_list = []
            for turni in range(group_num):
                temp_code_list.append(
                    factor_merge[factor_merge['quantile'] == turni + 1][
                        f'{self.asset_type}code'].tolist())
            [turn_over_store_list[idx].append(value) for idx, value in enumerate(temp_code_list)]
            daterecord.append(adjusting_date_list[datenum])
        ##########计算换手率
        allstockchange = pd.DataFrame(turn_over_store_list).T
        allstockchange.index = daterecord
        # len(allstockchange)
        allstockchange_set = allstockchange.applymap(lambda x: set(x))
        # test len
        # num_date=allstockchange_set.applymap(lambda x: len(x))
        allstockchange_set_shift = allstockchange_set.shift(1)
        change_stock = (allstockchange_set - allstockchange_set_shift).dropna()
        allstockchange_set_shift_drop = allstockchange_set_shift.dropna()
        # num2_date=change_stock.applymap(lambda x: len(x))
        turn_over_ratio = change_stock.applymap(
            lambda x: len(x)) / allstockchange_set_shift_drop.applymap(
            lambda x: len(x))
        turn_over_ratio.index = pd.to_datetime(turn_over_ratio.index)
        turn_over_ratio.columns = ['分组' + str(x + 1) for x in turn_over_ratio.columns]
        #     ###记录换仓前各组证券代码
        #     _medium_code = medium_code
        #     _bottom_code = bottom_code
        #     _top_code = top_code
        #
        #     enddate = datenum + period + 1
        #     if enddate > len(distinct_trade_date):
        #         enddate = len(distinct_trade_date)
        #     # datenum=enddate
        #
        # allquant_simple = pd.DataFrame({'bottom': bottom_turnover, 'medium': medium_turnover, \
        #                                 'top': top_turnover}, index=daterecord)
        # ##画图
        # index = np.arange(len(allquant_simple['bottom']))
        # width = 0.3
        # plt.bar(left=index-width, height=allquant_simple['bottom'], width=width, color='deepskyblue', label=u'q20')
        # plt.bar(left=index, height=allquant_simple['medium'], width=width, color='powderblue', label=u'q50')
        # plt.bar(left=index + width, height=allquant_simple['top'], width=width, color='steelblue',
        #         label=u'q80')
        if plot:
            plt.figure(figsize=(16, 6))
            ax = turn_over_ratio.plot(figsize=(16, 20), layout=(5, 2), subplots=True, sharex=False)
            # for ii in range(group_num):
            #     plt.bar(pd.to_datetime(turn_over_ratio.index.tolist()), turn_over_ratio.iloc[:, ii],
            #             label='quantile' + str(ii + 1))
            plt.title(self.factor + ' 因子换手率')
            plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
            plt.grid(axis='y')
            plt.xticks(rotation=70)
            plt.show()

        return turn_over_ratio

    def ic_decline(self, decline_num, period='', freq='', rank=True):
        """
        ic 衰退计算函数，计算因子和不同滞后期的收益的IC，period 为每期间隔天数，decline_num为统计期数
        :param decline_num: 衰退统计期数
        :param period: 统计周期
        :param freq: 统计频率
        :param rank: 是否为秩相关系数
        :return:
        """
        if freq == '':
            freq = self.freq
        if period == '':
            period = self.period
        allPrice = self.all_stock_price.copy()
        factor_data = self.factor_data.copy()
        mergedata = allPrice.merge(factor_data, how='inner',
                                   on=['tradedate', f'{self.asset_type}code'])
        mergedata.drop_duplicates(inplace=True)
        adjusting_date_list = self.control_date_freq(period, freq)

        mergedata_group = mergedata.groupby('tradedate')

        # factor_series = mergedata_group.get_group(adjusting_date_list[0])
        IC_decline = pd.DataFrame()
        IC_decline_time = []

        IC_value_df = [[] for xx in range(decline_num)]
        if len(adjusting_date_list) - decline_num <= 1:
            print('衰退长度过长或时间不够')
            return
        for i in range(0, len(adjusting_date_list) - decline_num):
            # print(adjusting_date_list[i])
            IC_decline_value = []
            for ii in range(0, decline_num):
                # print(adjusting_date_list[i + ii + 1])
                begindate_series = mergedata_group.get_group(adjusting_date_list[i])
                enddate_series = mergedata_group.get_group(adjusting_date_list[i + ii + 1])

                period_return = begindate_series[
                    ['tradedate', f'{self.asset_type}code', self.factor, 'fap_close']].merge(
                    enddate_series[['tradedate', f'{self.asset_type}code', 'fap_close']],
                    how='inner',
                    on=f'{self.asset_type}code')
                period_return.drop_duplicates(inplace=True)
                period_return['period_return'] = period_return['fap_close_y'] / period_return[
                    'fap_close_x'] - 1

                IC_decline_value.append(
                    st.spearmanr(period_return[['period_return', self.factor]])[0] if rank else
                    st.pearsonr(period_return['period_return'], period_return[self.factor])[0])

            [IC_value_df[idx].append(value) for idx, value in enumerate(IC_decline_value)]
        IC_decline_value_mean = np.mean(IC_value_df, axis=1)
        num_list = list(range(1, decline_num + 1))
        name_list = list(map(lambda x: 'period' + str(x), num_list))
        plt.bar(range(len(IC_decline_value_mean)), IC_decline_value)
        plt.xticks(range(len(IC_decline_value_mean)), name_list)
        plt.title(self.factor + '因子IC衰退')
        plt.grid(axis='y')
        plt.show()
        return IC_decline_value_mean

    def ic_ir(self, period='', freq='', rank=True, winsor=True, standard=True, neutral=False):
        """
        period 为每期间隔天数，decline_num为统计期数

        :param period: 统计周期
        :param freq: 统计频率
        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :param rank: 是否为秩相关系数
        :return:
        """

        if freq == '':
            freq = self.freq
        if period == '':
            period = self.period
        adjusting_date_list = self.control_date_freq(period, freq)
        allPrice = self.all_stock_price.copy()
        factor_data = self.factor_data.copy()
        industry = self.industry_information.copy()
        distinct_trade_date = allPrice['tradedate'].sort_values().unique()
        mergedata = (allPrice.merge(industry, how='inner', on=f'{self.asset_type}code')).merge(
            factor_data,
            how='inner',
            on=['tradedate',
                f'{self.asset_type}code'])
        mergedata.drop_duplicates(inplace=True)
        industry_mergedata_group = mergedata.groupby('sw_ind1')
        unique_industry = mergedata['sw_ind1'].unique()
        IC_industry = []
        IC_value = []
        for i in range(len(unique_industry)):  # 遍历所有行业
            industry_mergedata_date_group = industry_mergedata_group.get_group(unique_industry[i])
            mergedata_industry_group = industry_mergedata_date_group.groupby('tradedate')

            for ii in range(0, len(adjusting_date_list)):  # 遍历每个行业每一期
                if ii >= (len(adjusting_date_list) - 1):
                    break
                IC_industry.append(unique_industry[i])
                begindate_series = mergedata_industry_group.get_group(adjusting_date_list[ii])
                begindate_series = begindate_series.set_index(f'{self.asset_type}code')
                begindate_series = fp.preprocess(begindate_series, self.factor, self.asset_type,
                                                 adjusting_date_list[ii],
                                                 winsor, neutral,
                                                 standard, self.exclude_industry,
                                                 self.exclude_style)
                begindate_series.reset_index(inplace=True)
                enddate_series = mergedata_industry_group.get_group(adjusting_date_list[(ii + 1)])

                period_return = begindate_series[
                    ['tradedate', f'{self.asset_type}code', 'fap_close', self.factor]].merge(
                    enddate_series[['tradedate', f'{self.asset_type}code', 'fap_close']],
                    how='inner',
                    on=f'{self.asset_type}code')
                period_return.drop_duplicates(inplace=True)
                period_return['period_return'] = period_return['fap_close_y'] / period_return[
                    'fap_close_x'] - 1

                IC_value.append(
                    st.spearmanr(period_return[['period_return', self.factor]])[0] if rank else
                    st.pearsonr(period_return['period_return'], period_return[self.factor])[0])

        industry_IC_df = pd.DataFrame({'industry': IC_industry, 'IC': IC_value})

        ICIR_df = industry_IC_df.groupby('industry').mean() / industry_IC_df.groupby(
            'industry').std()
        ICIR_df.columns = ['ICIR']
        # ICIR_df=ICIR_df.reset_index()

        ICIR_df = ICIR_df['ICIR'].sort_values(ascending=False)

        rects = plt.bar(range(len(ICIR_df)), ICIR_df)
        plt.xticks(range(len(ICIR_df)), ICIR_df.index.tolist())
        plt.xticks(rotation=70)
        plt.title('区间内' + self.factor + '因子各行业ICIR值')
        plt.grid(axis='y')
        plt.show()
        return ICIR_df

    def factor_cummulate_return(self, winsor=True, neutral=False, standard=True, plot=True):
        """
        单因子累计收益率计算函数，回归方法。回归系数+1累乘

        :param winsor: 是否去极值
        :param neutral: 是否中性化
        :param standard: 是否标准化
        :return:
        """

        all_price = self.all_stock_price.copy()
        factor_data = self.factor_data.copy()
        temp_all_price = all_price.set_index(['tradedate', f'{self.asset_type}code'])[
            'fap_close'].unstack()

        stack_temp_all_price = temp_all_price.pct_change(1).shift(-1).stack()  # 算出当天因子对应的下一天收益
        all_price_next_day = stack_temp_all_price.reset_index()
        all_price_next_day.columns = ['tradedate', f'{self.asset_type}code', 'next_day_return']
        next_return_factor = all_price_next_day.merge(factor_data, how='inner',
                                                      on=['tradedate',
                                                          f'{self.asset_type}code'])  # 当天因子数据和下一天的收益
        return_factor_group = next_return_factor.groupby('tradedate')
        date_list = next_return_factor['tradedate'].unique().tolist()
        date_list2 = all_price['tradedate'].unique().tolist()

        factor_return_daily = []
        factor_return_date = []
        for datei in range(len(date_list)):
            # 资产池
            # print(date_list[datei])
            Univ = self.stock_pool_group.get_group(date_list[datei])
            # Univ = set_universe(self.universe, date_list[datei])
            return_factor_day = return_factor_group.get_group(date_list[datei])
            return_factor_day = return_factor_day.merge(Univ, how='inner',
                                                        on=f'{self.asset_type}code')
            return_factor_day.drop_duplicates(inplace=True)
            return_factor_day.set_index(f'{self.asset_type}code', inplace=True)
            return_factor_day = fp.preprocess(return_factor_day, self.factor, self.asset_type,
                                              date_list[datei],
                                              winsor, neutral,
                                              standard, self.exclude_industry, self.exclude_style)
            return_factor_day.reset_index(inplace=True)
            ##公式求解beta
            factor_beta = ((return_factor_day[self.factor] * return_factor_day[
                'next_day_return'].astype(
                float)).sum()) / ((return_factor_day[
                                       self.factor] ** 2).sum())
            factor_return_date.append(date_list2[datei + 1])
            factor_return_daily.append(factor_beta)

        factor_return = (pd.Series(factor_return_daily) + 1).cumprod()
        if plot:
            plt.figure(figsize=(16, 6))
            plt.plot(pd.to_datetime(factor_return_date), factor_return)
            plt.title('因子累计收益率')
            plt.grid(axis='y')
            plt.show()
        return factor_return

    @count_time
    def get_stock_industry_group_return(self, group_num, cut_type='quantile', plot=True):
        if self.stock_pool_group:
            adjusting_date_list = self.control_date_freq(self.period, freq=self.freq)  # 获取调仓日期
            dynamic_industry = pd.read_sql(f"select stockcode, tradedate, sw_ind1 "
                                           f"from stk_fac_ind_sw_ind1 "
                                           f"where tradedate <= '{self.end_date}' "
                                           f"and tradedate >= '{self.begin_date}' "
                                           f"and stockcode in {tuple(self.asset_pool)} ",
                                           self.con_factor)
            industry_names = pd.read_sql(f"select distinct sw_ind1 "
                                         f"from stk_fac_ind_sw_ind1 "
                                         f"where tradedate <= '{self.end_date}'"
                                         f"and tradedate >= '{self.begin_date}' ", self.con_factor)[
                'sw_ind1'].tolist()
            groups = [float(i + 1) for i in range(group_num)]
            full_index_list = [industry_names, groups]
            full_industry_group = pd.MultiIndex.from_product(full_index_list, names=['sw_ind1',
                                                                                     'group'])  # 构建全行业全分组索引

            dynamic_industry['tradedate'] = dynamic_industry['tradedate'].astype(str)
            freq_factor = self.factor_data.set_index(['tradedate', 'stockcode'])[
                self.factor].unstack().reindex(adjusting_date_list, method='ffill')  # 根据调仓日期调整因子值频率
            daily_ffill_factor = freq_factor.reindex(self.trade_date_list,
                                                     method='ffill').stack().rename(
                self.factor).reset_index()  # 转换回日频数据
            industry_factor = dynamic_industry.merge(daily_ffill_factor,
                                                     on=['stockcode', 'tradedate'], how='left')
            stock_return = self.all_stock_price.set_index(['tradedate', 'stockcode'])[
                'fap_close'].unstack().ffill().pct_change()
            stock_return = stock_return.stack().rename('ret').reset_index()
            industry_factor_return = stock_return.merge(industry_factor,
                                                        on=['stockcode', 'tradedate'], how='left')
            industry_factor_return['group'] = \
                industry_factor_return.groupby(['tradedate', 'sw_ind1'])[self.factor].apply(
                    fp.my_qcut2, group_num=group_num, cut_type=cut_type)
            industry_factor_return['group'] = industry_factor_return['group'].where(
                industry_factor_return['group'].isin(
                    [float(i + 1) for i in range(group_num)]))  # 分组失败的替换为nan值
            industry_group_tradedate_return = \
                industry_factor_return.groupby(['tradedate', 'sw_ind1', 'group'])['ret'].mean()
            industry_group_tradedate_return_un = industry_group_tradedate_return.unstack(
                level=[-2, -1])
            cum_ret = ((industry_group_tradedate_return_un + 1).cumprod() - 1).iloc[-1]
            cum_ret = cum_ret.reindex(full_industry_group).reset_index()  # 没有分组收益的填充为nan
            cum_ret.columns = ['sw_ind1', 'group', 'cum_ret']
            if plot:
                df_plot = cum_ret.set_index(['sw_ind1', 'group'])['cum_ret'].unstack()
                df_plot.plot(kind='bar', figsize=(16, 9))
                plt.show()
            return cum_ret
        else:
            print("目前只有股票资产支持行业分组收益")
            return


if __name__ == '__main__':
    ##############################因子分析外部数据使用说明###################################
    # 1. 列名约束
    # 外部数据最后一列为因子数据，时间轴列名为tradedate，其余列根据分析标的的不同类型有所差别。
    # 1.1 股票数据的标准列名为：tradedate(时间轴序列）, stockcode(股票代码序列）, factor_name(因子暴露序列）
    # 1.2 基金数据的标准列名为：tradedate(时间轴序列）, fundcode(股票代码序列）, factor_name(因子暴露序列）
    # 1.3 基金经理数据的标准列名为：tradedate(时间轴序列）, manager_code(基金经理代码序列）, fund_type(基金类型序列）, factor_name(因子暴露序列）
    # 2. 数据类型及格式约束
    # 外部数据的数据类型及格式约束如下：
    # 2.1 tradedate 为字符串，格式为YYYY-MM-DD
    # 2.2 stockcode,fundcode,manager_code,fund_type 为字符串
    # 2.3 factor_name 为float
    # 3. 数据频率
    # 数据频率统一为日频
    period_start = '2020-05-20'
    period_end = '2020-10-29'

    # %%
    # 外部数据测试用例
    FactorAnalyser = FactorAnalyseMachine(factor_name='etop',  # 分析因子名称
                                          begin_date=period_start,  # 分析开始时间
                                          end_date=period_end,  # 分析结束时间
                                          external_data=False,
                                          asset_type='stock',
                                          direction=[1])
    # 股票因子测试用例
    # 部分可用于测试的股票因子举例

    # 因子方向正向为1，负向为-1，
    # 多个因子请按顺序输入1和-1组成的list，
    # 如[1,1,-1]表示三个因子，其中第三个因子为负向
    FactorAnalyser.param_setting(period='monthly',  # 调仓周期
                                 universe='HS300',  # 标的池仅对股票有效，基金和基金经理对应标的暂定为上证综指
                                 benchmark='000300',  # 基准代码，如果输入avg则表示所有资产池的平均值
                                 freq=1,  # 调仓频率
                                 bool_exclude_industry=True,  # 中性化是否剔除行业因子，仅对股票有效
                                 list_exclude_style=[])  # 中性化是否剔除风格因子，仅对股票有效
    # FactorAnalyser.initial_data(winsor=True, standard=True,method='IR', window=5,
    #                          orthogonalize=True)  # 数据初始化

    FactorAnalyser.initial_data(winsor=True, standard=True, method='IR', window=5,
                                orthogonalize=True)  # 数据初始化

    # FactorAnalyser.describe_factor(date='', winsor=True, neutral=True, standard=True)  # 因子分布
    ICstatsdf, long_short_df, group_return_summary, group_return = \
        FactorAnalyser.longshort_portfolio(
            freq=1, group_num=5, winsor=True, neutral=False, standard=True,
            abnormal_threshold=0.11)  # 因子多空组合的净值曲线
    # time_ = dt.datetime.now()
    # print('总耗时', time_ - _time)

    # test = factoranalyser.factorturnover_simple(freq='', winsor=true, neutral=false, standard=true, group_num=10)
    # factoranalyser.icdecline(freq='', rank=true, decline_num=5)
    # # factoranalyser.icir(freq='', winsor=true, standard=true, neutral=false, rank=true)
    # factoranalyser.factor_cummulate_return(winsor=true, neutral=false, standard=true)
